#-*- coding:utf-8 -*-
from scrapy.spiders import CrawlSpider,Request
from data5u.items import Data5UItem
from scrapy.selector import Selector
from test import test_Proxy
import redis
import time

sleepTime = 0
fr = open('set.conf','r')
for line in fr:
    if line.find('TIME') != -1:
        sleepTime = int(line.strip().split('=')[1])
print sleepTime

r=redis.StrictRedis(host='localhost', port=3888, db=0, password='Los15061053721')
class Data5u(CrawlSpider):
    name='Data5uSpiders_1'
    allowed_domains=[]
    start_url='http://www.data5u.com/'   
    def start_requests(self):
        yield Request(self.start_url,self.data5u_parse,dont_filter=True)
#         time.sleep(sleepTime)
    
    def data5u_parse(self,response):
        sel = Selector(response)
        f = open('item.json','a+')
        for i in range(2,22):
            data_list = []
            ip = sel.xpath('/html/body/div[4]/li[2]/ul[%s]/span[1]/li/text()' % str(i)).extract()[0]
            port = sel.xpath('/html/body/div[4]/li[2]/ul[%s]/span[2]/li/text()' % str(i)).extract()[0]
            Anonymity = sel.xpath('/html/body/div[4]/li[2]/ul[%s]/span[3]/li/a/text()' % str(i)).extract()[0]
            Type = sel.xpath('/html/body/div[4]/li[2]/ul[%s]/span[4]/li/a/text()' % str(i)).extract()[0]
            Country = sel.xpath('/html/body/div[4]/li[2]/ul[%s]/span[5]/li/a/text()' % str(i)).extract()[0]
            PaC_list = sel.xpath('/html/body/div[4]/li[2]/ul[%s]/span[6]/li/a/text()' % str(i)).extract()
            if PaC_list != [] and len(PaC_list) ==2 :
                PaC = PaC_list[0]+PaC_list[1]
            elif len(PaC_list) == 1:
                PaC = PaC_list[0]
            else:
                PaC = ' '
            Operator_list = sel.xpath('/html/body/div[4]/li[2]/ul[%s]/span[7]/li/a/text()' % str(i)).extract()
            if Operator_list != []:
                Operator = Operator_list[0]
            else:
                Operator = ' '
            Speed = sel.xpath('/html/body/div[4]/li[2]/ul[%s]/span[8]/li/text()' % str(i)).extract()[0]
            try:
                Time = sel.xpath('/html/body/div[4]/li[2]/ul[%s]/span[9]/li/text()' % str(i)).extract()[0]
            except:
                pass
            
            ip_port = ip+':'+port
            print ip_port
            print test_Proxy(ip_port)
            if test_Proxy(ip_port) == True:
                data_list.append(ip_port) 
                data_list.append(Anonymity)
                data_list.append(Type)
                data_list.append(Country)
                data_list.append(PaC)
                data_list.append(Operator)
                data_list.append(Speed)
                data_list.append(Time)
                r.set(ip,data_list)
                for itemList in data_list:
                    f.write(itemList.encode('utf-8')+' ')
                f.write('\n')
        f.write('\n'+"Time : %s" % time.ctime()+'\n')
#         time.sleep(sleepTime)
        yield Request(self.start_url,callback=self.data5u_parse,dont_filter=True)
        